{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSA on toy dataset\n",
    "\n",
    "Let us look at a toy dataset. Rows correspond to documents. Columns correspond to terms. Each cell\n",
    "contains the term frequency. The terms “Gun” and “Violence” occur equal number of times in most documents, indicating clear correlation\n",
    "\n",
    "\\begin{vmatrix}\n",
    "  & violence & gun & america & roses \\\\\n",
    "  d_{0} & 0 & 0 & 0 & 2 \\\\\n",
    "  d_{1} & 1 & 1 & 1 & 0 \\\\\n",
    "  d_{2} & 2 & 2 & 0 & 0 \\\\\n",
    "  d_{3} & 3 & 3 & 0 & 0 \\\\\n",
    "  d_{4} & 5 & 5 & 0 & 0 \\\\\n",
    "  d_{5} & 0 & 1 & 0 & 0 \\\\\n",
    "  d_{6} & 1 & 0 & 0 & 0 \\\\\n",
    "\\end{vmatrix}\n",
    "\n",
    "Cosine similarity between document vectors is often used to measure similarity between two documents. Cosine Similarity only considers direct overlap of terms. The terms \"Gun\" and \"violence\" have clear correlation (they appear together in many other documents, so documents containing \"Gun\" should be similar to documents containing \"violence\"). Cosine Similarity will not see that. LSA wiil. In LSA terms often occuring together become part of the same topic. Documents are projected into topic space - e.g., \"Gun-violence\" is a topic - where indirect similarities are visible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "terms = [\"violence\", \"gun\", \"america\", \"roses\"]\n",
    "doc_term_matrix = torch.tensor([[0, 0, 0, 2], [1, 1, 1, 0], [2, 2, 0, 0], [3, 3, 0, 0], [5, 5, 0, 0], [0, 1, 0, 0], [1, 0, 0, 0]]).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Principal Values. 8.89 2.00 1.00 0.99\n"
     ]
    }
   ],
   "source": [
    "# Let us perform SVD\n",
    "U, S, V_t = torch.linalg.svd(doc_term_matrix)\n",
    "\n",
    "\n",
    "print(\"Principal Values. %0.2f %0.2f %0.2f %0.2f\"%(S[0], S[1], S[2], S[3]))\n",
    "\n",
    "# The columns of V are the topic vectors. Each topic vector can\n",
    "# be seen as a weighted sum of the terms in our vocabulary.\n",
    "V = V_t.T\n",
    "\n",
    "\n",
    "# Let us reduce this to a lower rank representation.\n",
    "# There is a big  drop in principal value from S[0] to S[1]. \n",
    "# Hence, we choose to cutoff all principal vectors beyong V[0].\n",
    "# We will retain only the first column of V, the principal axis. \n",
    "rank = 1\n",
    "U = U[:, :rank]\n",
    "V = V[:, :rank]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('violence', tensor(-0.7070)), ('gun', tensor(-0.7070)), ('america', tensor(-0.0181)), ('roses', tensor(1.1381e-09))]\n"
     ]
    }
   ],
   "source": [
    "# Now that we have reduced the dimensionality to only contain one topic, let \n",
    "# let us look at the weighted contributions of terms to this topic.\n",
    "term_topic_affinity = list(zip(terms, V[:, 0]))\n",
    "\n",
    "# Note that both violence and gun have every high affinity, and contribute equally to this\n",
    "# topic.\n",
    "print(term_topic_affinity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cosine similarity between document 5 and document 6 in original space is 0.0\n",
      "LSA topic based Cosine similarity between document 5 and document 6 is 1.0\n"
     ]
    }
   ],
   "source": [
    "# Let us consider 2 documents d5 and d6. \n",
    "\n",
    "def cosine_similarity(vec_1, vec_2):\n",
    "    return torch.dot(vec_1, vec_2) / (torch.linalg.norm(vec_1) * torch.linalg.norm(vec_2))\n",
    "\n",
    "\n",
    "# Note that the similarity between the two documents is 0 even though\n",
    "# intuitively they are similar\n",
    "d5_d6_similarity = cosine_similarity(doc_term_matrix[5], doc_term_matrix[6])\n",
    "assert d5_d6_similarity == 0\n",
    "print(\"Cosine similarity between document 5 and document 6 in original space is {}\".format(d5_d6_similarity))\n",
    "\n",
    "# Now let us instead look at the document representation in the topic space\n",
    "# We notice in this new space, documents 5 and 6 are close.\n",
    "doc_topic_matrix = torch.matmul(doc_term_matrix, V)\n",
    "d5_d6_similarity = cosine_similarity(doc_topic_matrix[5], doc_topic_matrix[6])\n",
    "print(\"LSA topic based Cosine similarity between document 5 and document 6 is {}\".format(d5_d6_similarity))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
